import pandas as pd
import numpy as np


# ['android_id', 'apptype', 'carrier', 'dev_height', 'dev_ppi',
#        'dev_width', 'label', 'lan', 'media_id', 'ntt', 'os', 'osv', 'package',
#        'sid', 'timestamp', 'version', 'fea_hash', 'location', 'fea1_hash',
#        'cus_type']
def preprocess(data_path, remove_list=['sid', 'os']):
    # data_path = r'datasets/train.csv'
    data = pd.read_csv(data_path)
    # remove id
    data = data.iloc[:, 1:]
    # Index(['lan', 'os', 'osv', 'version', 'fea_hash'], dtype='object')

    features = data.columns.tolist()
    if 'label' in features:
        label = data['label']
        features.remove('label')
        is_train = True
    else:
        is_train = False
        label = None
    sid = data['sid']
    # Remove features by columns
    col = features
    for i in remove_list:
        col.remove(i)

    data = data[col]
    # 构造fea_hash_len特征
    data['fea_hash_len'] = data['fea_hash'].map(lambda x: len(str(x)))
    data['fea1_hash_len'] = data['fea1_hash'].map(lambda x: len(str(x)))
    # Thinking：为什么将很大的，很长的fea_hash化为0？
    # 如果fea_hash很长，都归为0，否则为自己的本身
    data['fea_hash'] = data['fea_hash'].map(lambda x: 0 if len(str(x)) > 16 else int(x))
    data['fea1_hash'] = data['fea1_hash'].map(lambda x: 0 if len(str(x)) > 16 else int(x))

    # Timestamp transfer
    time = pd.to_datetime(data['timestamp'], unit='ms')
    temp = pd.DatetimeIndex(time)
    data['year'] = temp.year
    data['month'] = temp.month
    data['day'] = temp.day
    data['week_day'] = temp.weekday
    data['hour'] = temp.hour
    data['minute'] = temp.minute

    # osv
    import re
    def f(x):
        # print("x",x,type(x))
        l_v = 0
        m_v = 0
        s_v = 0
        if isinstance(x, str):
            it = re.finditer(r"\d[\d|\.]?", x)

            for i, match in enumerate(it):
                if i == 0:
                    l_v = match.group().replace('.', '')
                elif i == 1:
                    m_v = match.group().replace('.', '')
                else:
                    s_v = match.group()
            return l_v, m_v, s_v
        else:
            return l_v, m_v, s_v

    data['osv'] = data['osv'].apply(f, args=())
    # a_values = a.values()
    data[['osv_l_v', 'osv_m_v', 'osv_s_v']] = data['osv'].apply(pd.Series).astype(np.int)

    # Screen
    data['dev_height'][data.dev_height == 0] = 1.0
    data['dev_width'][data.dev_width == 0] = 1.0
    data['dev_ppi'][data.dev_ppi == 0] = 1.0
    data['dev_height'] = data['dev_height'].astype('float')
    data['dev_width'] = data['dev_width'].astype('float')
    data['dev_ppi'] = data['dev_ppi'].astype('float')
    data['dev_height_width_rate'] = data['dev_height'] / data['dev_width']
    data['dev_area'] = data['dev_height'] * data['dev_width']
    data['dev_area_ppi_rate'] = data['dev_area'] / data['dev_ppi']

    #lan
    lan_dict = {
        "zh-CN": 0,
        "zh": 1,
        "cn": 2,
        "zh_CN": 3,
        "Zh-CN": 4,
        "zh-cn": 5,
        "ZH": 6,
        "CN": 7,
        "tw": 8,
        "en": 9,
        "zh_CN_#Hans": 10,
        "ko": 11,
        "zh-TW": 12,
        "zh-HK": 13,
        "en-US": 14,
        "ja": 15,
        "en-GB": 16,
        "it": 17,
        "TW": 18,
        "mi": 19,
        "zh-MO": 20,
        "en_US":21,
        "zh-US":22,
        "in_ID":23,
        "miss": 24,
    }

    def f(x):
        # 判断是否在关键特征值里，是1，否0
        return lan_dict[x]

    data['lan'].fillna('miss', inplace=True)
    data['lan_no'] = data['lan'].apply(f, args=())

    return data, label, sid
